import os
from news_regular_expression import RegularExpression
from html_parser import parse_html

import requests

getting_news = "E:\\myPyFun\\getting_news"

url_list = [{'title': 'cls',            'name': '财联社',              'url': 'https://www.cls.cn/'},
            {'title': 'zaobao',         'name': '新加坡《联合早报》',     'url': 'https://www.zaobao.com/'},
            {'title': 'sputniknews',    'name': '俄罗斯卫星通讯社',      'url': 'https://sputniknews.cn/'}
            # {'name': '参考消息', 'url': 'https://www.cankaoxiaoxi.com/?t=62swu#/index'},
            # {'name': '界面新闻', 'url': 'https://www.jiemian.com/'},
            # {'name': '环球网', 'url': 'https://m.huanqiu.com/'},
            # {'name': '联合国新闻网', 'url': 'https://news.un.org/zh/'},
            # {'name': '财新网', 'url': 'https://www.caixin.com/?HOLDZH'},
            # {'name': '中国香港大公网', 'url': 'http://www.takungpao.com.hk/'},
            # {'name': '中国香港文汇网', 'url': 'https://www.wenweipo.com/'},
            # {'name': '中国澳门日报', 'url': 'http://www.macaodaily.com/html/2023-06/29/node_2.htm'},
            # {'name': '中国台湾中时新闻网', 'url': 'https://www.chinatimes.com/?chdtv'},
            # {'name': '日本共同网', 'url': 'https://china.kyodonews.net/'},
            # {'name': '日经中文网', 'url': 'https://d9shhjt4p7ouc.cloudfront.net/index.php'},
            # {'name': '韩联社', 'url': 'https://cn.yna.co.kr/'},
            # {'name': '韩国《中央日报》', 'url': 'https://chinese.joins.com/'},
            # {'name': '韩国《东亚日报》', 'url': 'https://www.donga.com/cn'},
            # {'name': '卡塔尔半岛电视台中文网', 'url': 'https://chinese.aljazeera.net/'},
            # {'name': '美国广播公司（ABC）', 'url': 'https://abc.com'},
            # {'name': '美国有线电视新闻网（CNN）', 'url': 'https://www.cnn.com'},
            # {'name': '美国消费者新闻与商业频道（CNBC）', 'url': 'https://www.cnbc.com/world/?region=world'},
            # {'name': '美联社', 'url': 'https://apnews.com/'},
            # {'name': '英国FT中文网', 'url': 'http://ftchineselive.com/'},
            # {'name': '法国《欧洲时报》', 'url': 'http://www.oushinet.com/'},
            # {'name': '（国际组织）世界贸易组织(WTO)', 'url': 'https://www.wto.org'},
            # {'name': '（国际组织）世界银行', 'url': 'https://www.shihang.org/zh/home'},
            # {'name': '（国际组织）国际货币基金组织（IMF）', 'url': 'https://www.imf.org/en/Home'},
            # {'name': '（国际组织）今日疫情新闻', 'url': 'http://outbreaknewstoday.com'},
            # {'name': '（国际组织）欧盟', 'url': 'https://commission.europa.eu/index_de'},
            # {'name': '（国际组织）东盟', 'url': 'https://asean.org'},
            # {'name': '中国香港《中华时报》', 'url': 'https://chinatimes.com.hk/'},
            # {'name': '中国香港星岛环球网', 'url': 'https://www.stnn.cc/'},
            # {'name': '中国香港商报', 'url': 'http://www.hkcd.com.hk'},
            # {'name': '俄罗斯《独立报》', 'url': 'https://www.ng.ru/'},
            # {'name': '今日俄罗斯RT ', 'url': 'https://www.rt.com/'},
            # {'name': '透视俄罗斯', 'url': 'http://tsrus.cn/'},
            # {'name': '日本华侨报', 'url': 'http://www.jnocnews.co.jp/'},
            # {'name': '日本NHK中文广播', 'url': 'https://www3.nhk.or.jp/nhkworld/zh/news/'},
            # {'name': '韩国《韩民族日报》', 'url': 'https://china.hani.co.kr/'},
            # {'name': '韩国《先驱者报》', 'url': 'https://www.koreaherald.com/index.php'},
            # {'name': '韩国经济新闻', 'url': 'https://www.kedglobal.com/'},
            # {'name': '韩国Naver搜索引擎 ', 'url': 'https://www.naver.com/'},
            # {'name': '韩国每日经济', 'url': 'https://www.mk.co.kr/'},
            # {'name': '韩国商业写新闻《BusinessKorea》', 'url': 'http://www.businesskorea.co.kr/'},
            # {'name': '印尼《国际日报》', 'url': 'https://guojiribao.com/'},
            # {'name': '泰国《星暹日报》', 'url': 'http://www.singsianyerpao.com/'},
            # {'name': '菲律宾《世界日报》', 'url': 'https://worldnews.net.ph/'},
            # {'name': '印尼《千岛日报》', 'url': 'https://www.qiandaoribao.com/'},
            # {'name': '吉尔吉斯斯坦国家通讯社 ', 'url': 'http://cn.kabar.kg/'},
            # {'name': '哈萨克斯坦国际通讯社', 'url': 'https://www.inform.kz/cn'},
            # {'name': '白俄罗斯国家通讯社', 'url': 'https://chn.belta.by/our-companyn/'},
            # {'name': '越南之声', 'url': 'https://vovworld.vn/zh-CN.vov'},
            # {'name': '越南越通社', 'url': 'https://zh.vietnamplus.vn/'},
            # {'name': '（国际组织）绿色和平组织', 'url': 'http://www.greenpeace.org.cn'},
            # {'name': '美国哥伦比亚广播公司（CBS）', 'url': 'https://www.cbsnews.com'},
            # {'name': '美国《洛杉矶时报》', 'url': 'https://www.latimes.com/'},
            # {'name': '美国《芝家哥太阳报》', 'url': 'https://chicago.suntimes.com/'},
            # {'name': '今日美国', 'url': 'http://www.usatoday.com/'},
            # {'name': '美国侨报网', 'url': 'https://www.uschinapress.com/web/SZ/index.html'},
            # {'name': '德意志新闻社', 'url': 'https://www.dpa.com/de'},
            # {'name': '（境外NGO组织）中外对话', 'url': 'https://chinadialogueocean.net/?lang=zh-hans'},
            # {'name': '美国MSN财经', 'url': 'https://www.msn.com/zh-tw/money/moneyrealtime'},
            # {'name': '英国《卫报》', 'url': 'https://guardian.ng/'},
            # {'name': '英国《镜报》', 'url': 'https://www.mirror.co.uk/'},
            # {'name': '英国每日电讯报 ', 'url': 'http://www.telegraph.co.uk/'},
            # {'name': '荷兰国际集团（ING）',
            #  'url': 'https://think.ing.com/search/results?keywords=China&orderby_sort=date&range-to'
            #         ':entry_date=2023-06-28&require_all=category'},
            # {'name': '西班牙欧华网', 'url': 'http://www.ouhua.info/'},
            # {'name': '新西兰华页中文网', 'url': 'http://www.mpages.co.nz/'},
            # {'name': '新西兰中文先驱网', 'url': 'https://www.chineseherald.co.nz/'},
            # {'name': '加拿大埃德蒙顿中文网', 'url': 'http://www.edmontonchina.ca/'},
            # {'name': '乌克兰国家通讯社', 'url': 'https://www.ukrinform.org/'},
            # {'name': '芬兰《Daily Finland》', 'url': 'https://www.dailyfinland.fi/business'}
            ]

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 "
                  "Safari/537.36"
}

regx = RegularExpression()


def do_search(u_name, u_url):
    try:
        r = requests.get(u_url, headers=headers, timeout=10)
        r.encoding = "UTF-8"
        print(r.url)
        print(r.json)
        u_name += ".html"
        html_file = os.path.join(getting_news, "html", u_name)
        with open(html_file, "w", encoding="UTF-8") as f:
            f.write(r.text)
            # f.write(r.content)
    except requests.RequestException:
        print(f"load {u_name} Time Out")


def welcome(name, ver):
    print(f"Hi, {name}" + "\r\n")
    print(f"Version: {ver}")


def load_url():
    _url = os.path.join("E:\\", "news", "oversea.txt")
    print(f"_url: {_url}")
    with open(_url, "r+", encoding="UTF-8") as url_f:
        url_lines = url_f.readlines()
        for url_l in url_lines:
            re_url = regx.URL_HEAD.search(url_l)
            if re_url:
                # print(re_url.group(0))
                print(re_url.group(1), re_url.group(2))
                url_dict = {"name": re_url.group(1), "url": "http" + re_url.group(2)}
                url_list.append(url_dict)

            else:
                print(f"re_url:{re_url}")

    print(f"url: {url_list}")


def remove_folder(folder_path):
    for root, dirs, files in os.walk(folder_path):
        for f in files:
            os.remove(os.path.join(root, f))
        for d in dirs:
            remove_folder(os.path.join(root, d))

    os.rmdir(folder_path)


if __name__ == '__main__':
    welcome('Getting News', "v1.00")
    html_path = os.path.join(getting_news, "html")
    if os.path.exists(html_path):
        remove_folder(html_path)
    os.makedirs(html_path)
    # load_url()
    for udict in url_list:
        do_search(udict["name"], udict["url"])
        # do_search("sputniknews", "https://sputniknews.cn/")
        parse_html(html_path, udict, "2024-02-04 16:00:00")
